import bs4
import traceback
import requests
from bs4 import BeautifulSoup
import re
import pandas as pd

import urllib.request

try:
    reg = "[^0-9A-Za-z\u4e00-\u9fa5]"
    sum=0

    nameList = []
    addressList = []
    focusList = []
    emailList = []
    urlList = []


    for x in range(51,99):
        a = str(x)
        url = "http://seea.tju.edu.cn/info/1013/15"+ a +".htm"
    # url = "http://seea.tju.edu.cn/info/1013/1596.htm"
        r = requests.get(url)
        if(r.status_code==200):
            if (r.text.strip() != ''):
                urlList.append(url)
                html=r.text.encode('iso-8859-1')
                bf=BeautifulSoup(html, 'lxml')

                text = bf.find_all(id = 'content_head')
                nameList.append(text[0].text.strip())

                addressOrigin = bf.find(text = re.compile('通讯地址'))
                
                if isinstance(addressOrigin, bs4.element.NavigableString):
                    addressStr = ''
                    

                    for addressTmp in addressOrigin.parent.next_siblings:
                        
                        if isinstance(addressTmp, bs4.element.NavigableString):
                            if(addressTmp.strip() != ''):
                                addressList.append(addressTmp)
                        elif isinstance(addressTmp, bs4.element.Tag):
                            for addressE in addressTmp:
                                addressStr += addressE.string

                        
                    if(addressStr.strip()!= ''):
                        addressList.append(addressStr)


                email = bf.find_all(text=re.compile('@[tju|126|163|gmail|qq|edu|outlook|hotmail|sina]'))


                emailList.append(email[0].text.strip())

                sc = ''
                focus = bf.find(text = re.compile('主要研究方向：'))
                if isinstance(focus, bs4.element.NavigableString):
                    flag=1
                    if(focus.parent.parent.name =='p'):
                        for search in focus.parent.parent.next_siblings:
                            if(flag):
                                if isinstance(search, bs4.element.Tag):
                                    for inner in search.contents:
                                        if(inner.string):
                                            if ('科研项目' in inner.string):
                                                flag=0
                                            else:
                                                sc += inner.string

                    elif(focus.parent.parent.parent.name =='p'):
                        for search in focus.parent.parent.parent.next_siblings:
                            
                            if(flag):
                                if isinstance(search, bs4.element.Tag):
                                    
                                    for inner in search.contents:
                                        if(inner.string):
                                            if ('科研项目' in inner.string):
                                                flag=0
                                            else:
                                                sc += inner.string
                    else:
                        sc = '未获取'
                sum = sum+1
                focusList.append(sc)
                print("爬取成功:",sum,"条")
        else:
            print("爬取失败: 资源不存在")
            continue


    result = {
        'name' : nameList,
        'url' : urlList,
        'address' :addressList,
        'email' : emailList,
        'focus' : focusList 
    }
    df = pd.DataFrame.from_dict(result, orient='index')

    df.to_excel("deo.xlsx",index=False)


    
except Exception:
    traceback.print_exc()
    print(url)
    print(r.status_code)
